<!DOCTYPE html>
<html>
<head><meta name="generator" content="Hexo 3.9.0">
  <!-- hexo-inject:begin --><!-- hexo-inject:end --><meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">



<title>
  
    Category: data analysis
  
</title>

<meta name="description" content="I am a current master at UPENN. This page serves as a record of my life. I may post study blogs, daily blogs, and personal opions. XD">
<meta property="og:type" content="website">
<meta property="og:title" content="Zexi &quot;Josh&quot; Jin">
<meta property="og:url" content="http://yoursite.com/categories/data-analysis/index.html">
<meta property="og:site_name" content="Zexi &quot;Josh&quot; Jin">
<meta property="og:description" content="I am a current master at UPENN. This page serves as a record of my life. I may post study blogs, daily blogs, and personal opions. XD">
<meta property="og:locale" content="en">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Zexi &quot;Josh&quot; Jin">
<meta name="twitter:description" content="I am a current master at UPENN. This page serves as a record of my life. I may post study blogs, daily blogs, and personal opions. XD">


  <link rel="alternative" href="/atom.xml" title="Zexi &#34;Josh&#34; Jin" type="application/atom+xml">



  <link rel="icon" href="/favicon.png">


<link rel="stylesheet" href="/perfect-scrollbar/css/perfect-scrollbar.min.css">
<link rel="stylesheet" href="/styles/main.css"><!-- hexo-inject:begin --><!-- hexo-inject:end -->






</head>
<body>
  <!-- hexo-inject:begin --><!-- hexo-inject:end --><div class="mobile-header">
  <button class="sidebar-toggle" type="button">
    <span class="icon-bar"></span>
    <span class="icon-bar"></span>
    <span class="icon-bar"></span>
  </button>
  <a class="title" href="/">Zexi &#34;Josh&#34; Jin</a>
</div>

  <div class="main-container">
    <div class="sidebar">
  <div class="header">
    <h1 class="title"><a href="/">Zexi &#34;Josh&#34; Jin</a></h1>
    
    <div class="info">
      <div class="content">
        
          <div class="description">I am a current master at UPENN. This page serves as a record of my life. I may post study blogs, daily blogs, and personal opions. XD</div>
        
        
      </div>
      
        <div class="avatar">
          
            <a href="/about"><img src="https://i.ibb.co/XXV2FLb/Screen-Shot-2019-10-17-at-12-31-49-PM.png"></a>
          
        </div>
      
    </div>
  </div>
  <div class="body">
    
      
        <ul class="nav">
          
            
              <li class="category-list-container">
                <a href="javascript:;">Category</a>
                <ul class="category-list"><li class="category-list-item"><a class="category-list-link" href="/categories/backend/">backend</a><span class="category-list-count">2</span></li><li class="category-list-item"><a class="category-list-link" href="/categories/data-analysis/">data analysis</a><span class="category-list-count">3</span></li><li class="category-list-item"><a class="category-list-link" href="/categories/data-structure/">data structure</a><span class="category-list-count">4</span></li><li class="category-list-item"><a class="category-list-link" href="/categories/photos/">photos</a><span class="category-list-count">2</span></li></ul>
              </li>
            
          
            
              <li class="tag-list-container">
                <a href="javascript:;">Tag</a>
                <ul class="tag-list"><li class="tag-list-item"><a class="tag-list-link" href="/tags/LeetCode/">LeetCode</a><span class="tag-list-count">3</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/algorithm/">algorithm</a><span class="tag-list-count">4</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/backend/">backend</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/backtrack/">backtrack</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/binary-tree/">binary tree</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/concurrency/">concurrency</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/django/">django</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/java/">java</a><span class="tag-list-count">4</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/new-york/">new york</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/pandas/">pandas</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/photos/">photos</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/python/">python</a><span class="tag-list-count">4</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/roc-auc/">roc-auc</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/sorting/">sorting</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/travel/">travel</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/traversal/">traversal</a><span class="tag-list-count">1</span></li></ul>
              </li>
            
          
            
              <li class="archive-list-container">
                <a href="javascript:;">Archive</a>
                <ul class="archive-list"><li class="archive-list-item"><a class="archive-list-link" href="/archives/2019/">2019</a><span class="archive-list-count">10</span></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2018/">2018</a><span class="archive-list-count">1</span></li></ul>
              </li>
            
          
        </ul>
      
        <ul class="nav">
          
            
              <li>
                <a href="/" title="Homepage">Homepage</a>
              </li>
            
          
            
              <li>
                <a href="/about" title="About">About</a>
              </li>
            
          
        </ul>
      
    
  </div>
</div>

    <div class="main-content">
      
        <div style="max-width: 1000px">
      
          <h1>data analysis</h1>


  
  
    <div class="post-list">
  

    <div class="post-list-item article">
      <h3 class="article-header">
        <a href="/2019/11/21/random-pandas-note-1/">
  Random Pandas Note 2
</a>

      </h3>
      

      <div class="article-info">
        <a href="/2019/11/21/random-pandas-note-1/"><span class="article-date">
  2019-11-21
</span>
</a>
        
	<span class="article-category tagcloud">
	<ul class="article-category-list"><li class="article-category-list-item"><a class="article-category-list-link" href="/categories/data-analysis/">data analysis</a></li></ul>
	</span>


        
	<span class="article-tag tagcloud">
		<ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/pandas/">pandas</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/python/">python</a></li></ul>
	</span>


      </div>
      <div class="article-entry">
        
          <p>Reference: <a href="https://www.mikulskibartosz.name/how-to-split-a-list-inside-a-dataframe-cell-into-rows-in-pandas/" target="_blank" rel="noopener">https://www.mikulskibartosz.name/how-to-split-a-list-inside-a-dataframe-cell-into-rows-in-pandas/</a></p>
<p>Given a table containing a column of lists, like:</p>
<div class="table-container">
<table>
<thead>
<tr>
<th style="text-align:center">id</th>
<th style="text-align:center">name</th>
<th style="text-align:center">tags</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:center">1</td>
<td style="text-align:center">winter squash</td>
<td style="text-align:center">[&#x2018;60-minutes&#x2019;, &#x2018;time-to-make&#x2019;, &#x2018;healthy&#x2019;]</td>
</tr>
<tr>
<td style="text-align:center">2</td>
<td style="text-align:center">braised pork</td>
<td style="text-align:center">[&#x2018;2-hours-more&#x2019;, &#x2018;time-to-make&#x2019;, &#x2018;chinese&#x2019;]</td>
</tr>
<tr>
<td style="text-align:center">3</td>
<td style="text-align:center">chilli Beef</td>
<td style="text-align:center">[&#x2018;4-hours-more&#x2019;, &#x2018;chinese&#x2019;]</td>
</tr>
</tbody>
</table>
</div>
<p>Like the &#x2018;unwind&#x2019; function in mongodb, to turn it into:</p>
<div class="table-container">
<table>
<thead>
<tr>
<th>id</th>
<th>name</th>
<th>tags</th>
</tr>
</thead>
<tbody>
<tr>
<td>1</td>
<td>winter squash</td>
<td>&#x2018;60-minutes&#x2019;</td>
</tr>
<tr>
<td>1</td>
<td>winter squash</td>
<td>&#x2018;time-to-make&#x2019;</td>
</tr>
<tr>
<td>1</td>
<td>winter squash</td>
<td>&#x2018;healthy&#x2019;</td>
</tr>
<tr>
<td>2</td>
<td>braised pork</td>
<td>&#x2018;2-hours-more&#x2019;</td>
</tr>
<tr>
<td>2</td>
<td>braised pork</td>
<td>&#x2018;time-to-make&#x2019;</td>
</tr>
<tr>
<td>2</td>
<td>braised pork</td>
<td>&#x2018;chinese&#x2019;</td>
</tr>
<tr>
<td>3</td>
<td>chilli Beef</td>
<td>&#x2018;4-hours-more&#x2019;</td>
</tr>
<tr>
<td>3</td>
<td>chilli Beef</td>
<td>&#x2018;chinese&#x2019;</td>
</tr>
</tbody>
</table>
</div>
<p>Here&#x2019;s who we do it:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">tags_df[&apos;tags&apos;] = tags_df.tags.apply(lambda x: x[1:-1].split(&apos;,&apos;))</span><br><span class="line">clean_tag_df = tags_df.tags.apply(pd.Series).merge(tags_df, right_index = True, left_index = True) \</span><br><span class="line">    .drop([&quot;tags&quot;], axis = 1) \</span><br><span class="line">    .melt(id_vars = [&apos;recipe_id&apos;], value_name = &quot;tags&quot;) \</span><br><span class="line">    .drop(&quot;variable&quot;, axis = 1).dropna()</span><br></pre></td></tr></table></figure>
<p>Breaking it down, we have the following to turn the tags from a string into a list of strings<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">tags_df[&apos;tags&apos;] = tags_df.tags.apply(lambda x: x[1:-1].split(&apos;,&apos;))</span><br></pre></td></tr></table></figure></p>
<p>Next step, turning a list of tags into multiple columns&#xFF1A;<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">tags_df.tags.apply(pd.Series)</span><br></pre></td></tr></table></figure></p>
<p>It turns the chart like:</p>
<div class="table-container">
<table>
<thead>
<tr>
<th>1</th>
<th>2</th>
<th>3</th>
</tr>
</thead>
<tbody>
<tr>
<td>&#x2018;60-minutes&#x2019;</td>
<td>&#x2018;time-to-make&#x2019;</td>
<td>&#x2018;healthy&#x2019; </td>
</tr>
<tr>
<td>&#x2018;2-hours-more&#x2019;</td>
<td>&#x2018;time-to-make&#x2019;</td>
<td>&#x2018;chinese&#x2019;</td>
</tr>
<tr>
<td>&#x2018;4-hours-more&#x2019;</td>
<td>&#x2018;chinese&#x2019;</td>
<td>NaN </td>
</tr>
</tbody>
</table>
</div>
<p>Then, we join tags with the rest of the list:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">prev_df.merge(tags_df, right_index = True, left_index = True)</span><br></pre></td></tr></table></figure></p>
<p>Then, we drop the duplicated &#x201C;tags&#x201D; column and unwind different columns of the tags into different rows:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">prev_df.drop([&quot;tags&quot;], axis = 1).melt(id_vars = [&apos;recipe_id&apos;], value_name = &quot;tags&quot;)</span><br></pre></td></tr></table></figure></p>
<p>Lastly, we remove the &#x201C;variable&#x201D; column, which we might not need:<br><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">prev_df.drop(&quot;variable&quot;, axis = 1).dropna()</span><br></pre></td></tr></table></figure></p>

        
      </div>
    </div>

  
    </div>
  

  
  
    <div class="post-list">
  

    <div class="post-list-item article">
      <h3 class="article-header">
        <a href="/2019/10/21/ROC-AUC-curve/">
  Confusion Matrix Summary
</a>

      </h3>
      
  <div class="article-gallery article-gallery-photos" itemscope itemtype="http://schema.org/ImageGallery">
    
      <figure class="article-gallery-img" itemprop="associatedMedia" itemscope itemtype="http://schema.org/ImageObject">
        <a href="https://i.ibb.co/F483Wfr/20160120112441421.png" itemprop="contentUrl" rel="gallery_ck3ah5hir002qaypqjsnis0w5" title style="background-image: url(https://i.ibb.co/F483Wfr/20160120112441421.png)">
          <img src="https://i.ibb.co/F483Wfr/20160120112441421.png" itemprop="thumbnail" alt>
        </a>
        <figcaption itemprop="caption description"></figcaption>
      </figure>
    
  </div>


      <div class="article-info">
        <a href="/2019/10/21/ROC-AUC-curve/"><span class="article-date">
  2019-10-21
</span>
</a>
        
	<span class="article-category tagcloud">
	<ul class="article-category-list"><li class="article-category-list-item"><a class="article-category-list-link" href="/categories/data-analysis/">data analysis</a></li></ul>
	</span>


        
	<span class="article-tag tagcloud">
		<ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/roc-auc/">roc-auc</a></li></ul>
	</span>


      </div>
      <div class="article-entry">
        
          <p>A simple cheat-sheet for confusion matrix.</p>

        
      </div>
    </div>

  
    </div>
  

  
  
    <div class="post-list">
  

    <div class="post-list-item article">
      <h3 class="article-header">
        <a href="/2019/10/19/random-pandas-note/">
  Random Pandas Note 1
</a>

      </h3>
      

      <div class="article-info">
        <a href="/2019/10/19/random-pandas-note/"><span class="article-date">
  2019-10-19
</span>
</a>
        
	<span class="article-category tagcloud">
	<ul class="article-category-list"><li class="article-category-list-item"><a class="article-category-list-link" href="/categories/data-analysis/">data analysis</a></li></ul>
	</span>


        
	<span class="article-tag tagcloud">
		<ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/pandas/">pandas</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/python/">python</a></li></ul>
	</span>


      </div>
      <div class="article-entry">
        
          <p>To change a row value based on more than one conditions, use the following:<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">df[<span class="string">&apos;A&apos;</span>] = np.where(((df[<span class="string">&apos;B&apos;</span>] == <span class="string">&apos;some_value&apos;</span>) &amp; (df[<span class="string">&apos;C&apos;</span>] == <span class="string">&apos;some_other_value&apos;</span>)), true_value, false_value)</span><br></pre></td></tr></table></figure></p>
<p>Another equally efficient approach is using loc:<br>Note that this method does not give a default value when the condition is not met. So, the same code might be required to run twice or using .fillna() method.<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">df.loc[(df.B == <span class="string">&apos;some_value&apos;</span>) | (df.C == <span class="string">&apos;some_other_value&apos;</span>), <span class="string">&apos;A&apos;</span>] = true_value</span><br></pre></td></tr></table></figure></p>
<p>ps: for &#x2018;and&#x2019; or &#x2018;or&#x2019; operation, use symbol &#x2018;&amp;&#x2019; or &#x2018;|&#x2019;.</p>

        
      </div>
    </div>

  
    </div>
  






          <div class="main-footer">
  
    © 2019 Zexi &#34;Josh&#34; Jin - Powered by <a href="http://hexo.io" target="_blank">Hexo</a> - Theme <a href="https://github.com/denjones/hexo-theme-chan" target="_blank">Chan</a>
  
</div>

      
        </div>
      
    </div>
  </div>
  <script src="//apps.bdimg.com/libs/jquery/2.1.4/jquery.min.js"></script>

  <link rel="stylesheet" href="/PhotoSwipe/photoswipe.css">
  <link rel="stylesheet" href="/PhotoSwipe/default-skin/default-skin.css">

  <!-- Root element of PhotoSwipe. Must have class pswp. -->
  <div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">

    <!-- Background of PhotoSwipe.
             It's a separate element as animating opacity is faster than rgba(). -->
    <div class="pswp__bg"></div>

    <!-- Slides wrapper with overflow:hidden. -->
    <div class="pswp__scroll-wrap">

      <!-- Container that holds slides.
                PhotoSwipe keeps only 3 of them in the DOM to save memory.
                Don't modify these 3 pswp__item elements, data is added later on. -->
      <div class="pswp__container">
        <div class="pswp__item"></div>
        <div class="pswp__item"></div>
        <div class="pswp__item"></div>
      </div>

      <!-- Default (PhotoSwipeUI_Default) interface on top of sliding area. Can be changed. -->
      <div class="pswp__ui pswp__ui--hidden">

        <div class="pswp__top-bar">

          <!--  Controls are self-explanatory. Order can be changed. -->

          <div class="pswp__counter"></div>

          <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>

          <button class="pswp__button pswp__button--share" title="Share"></button>

          <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>

          <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>

          <!-- Preloader demo http://codepen.io/dimsemenov/pen/yyBWoR -->
          <!-- element will get class pswp__preloader--active when preloader is running -->
          <div class="pswp__preloader">
            <div class="pswp__preloader__icn">
              <div class="pswp__preloader__cut">
                <div class="pswp__preloader__donut"></div>
              </div>
            </div>
          </div>
        </div>

        <div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
          <div class="pswp__share-tooltip"></div>
        </div>

        <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)"></button>

        <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)"></button>

        <div class="pswp__caption">
          <div class="pswp__caption__center"></div>
        </div>
      </div>
    </div>
  </div>

  <script src="/PhotoSwipe/photoswipe.js"></script>
  <script src="/PhotoSwipe/photoswipe-ui-default.js"></script>


<script src="/perfect-scrollbar/js/min/perfect-scrollbar.min.js"></script>
<script src="/scripts/main.js"></script><!-- hexo-inject:begin --><!-- hexo-inject:end -->

</body>
</html>
